#!/bin/bash
#
# Author:   Joshua Chen <iesugrace@gmail.com>
# Date:     2017-08-02 18:37:38
# Location: Shenzhen
# Desc: 
#

for num in "$1" "$2"
do
    if ! grep -qE '^[0-9]+$' <<< "$num"
    then
        echo "usage: $(basename $0) start-id task-amount" >&2
        exit 1
    fi
done

next=$1
count=$2
bs=10000
dir_prefix=spider_
feeder="proxy_feeder/feed_proxy_db.sh"
fetcher="./fetcher.sh"
wd=$(dirname $0)
cd "$wd"

#
# Won't start if old tasks exist
#
if test $(find -maxdepth 1 -name "${dir_prefix}*" | wc -l) -ne 0
then
    echo "old tasks exist, exiting."
    echo "you shall run resume script, or delete the old tasks."
    exit 1
fi

# start proxy feeder
echo "starting proxy feeder..."
$feeder &

# give some time for the feerder to acquire the lock
sleep 5

# start spiders
for i in $(seq $count)
do
    echo "starting spider number $i"
    newdir=${dir_prefix}$i
    cp -r lagou_fetcher $newdir
    cd $newdir
    $fetcher $next $bs &
    cd ..
    next=$((next + bs))
done
